#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
AutoML随机搜索策略示例
Random Search Strategy Example

本示例演示如何使用Auto-sklearn的随机搜索和ROAR算法进行超参数优化。
随机搜索提供了一种简单、透明、无偏的超参数优化方法，适用于快速原型验证和算法公平比较。

作者: AutoML学习团队
日期: 2024-07-12
"""

import os
import time
import warnings
from datetime import datetime
from pprint import pprint

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn.datasets
import sklearn.metrics
import sklearn.model_selection
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc
from scipy import stats

from smac.facade.roar_facade import ROAR
from smac.scenario.scenario import Scenario

import autosklearn.classification

# 忽略警告信息
warnings.filterwarnings('ignore')

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False


class RandomSearchController:
    """
    随机搜索策略控制器
    
    管理Auto-sklearn的随机搜索和ROAR算法，提供：
    1. 纯随机搜索策略
    2. ROAR (Random Online Aggressive Racing) 算法
    3. 搜索过程监控和分析
    4. 性能对比评估
    """
    
    def __init__(self, time_budget=120, search_budget=50):
        """
        初始化随机搜索控制器
        
        参数:
            time_budget (int): 总时间预算（秒）
            search_budget (int): 搜索预算（评估次数）
        """
        self.time_budget = time_budget
        self.search_budget = search_budget
        self.search_history = []
        self.performance_tracker = PerformanceTracker()
        
        print(f"🎲 初始化随机搜索控制器")
        print(f"   时间预算: {time_budget}秒")
        print(f"   搜索预算: {search_budget}次评估")
    
    def get_roar_object_callback(self):
        """
        获取ROAR算法回调函数
        
        返回:
            function: ROAR优化器回调函数
        """
        def roar_callback(scenario_dict, seed, ta, ta_kwargs, 
                         metalearning_configurations, n_jobs, dask_client,
                         multi_objective_algorithm, multi_objective_kwargs):
            """ROAR算法回调函数"""
            
            if n_jobs > 1 or (dask_client and len(dask_client.nthreads()) > 1):
                raise ValueError(
                    "请确保代码在 `if __name__ == '__main__'` 保护下运行"
                )
            
            scenario = Scenario(scenario_dict)
            return ROAR(
                scenario=scenario,
                rng=seed,
                tae_runner=ta,
                tae_runner_kwargs=ta_kwargs,
                run_id=seed,
                dask_client=dask_client,
                n_jobs=n_jobs,
            )
        
        return roar_callback
    
    def get_random_search_callback(self):
        """
        获取随机搜索回调函数
        
        返回:
            function: 随机搜索优化器回调函数
        """
        def random_search_callback(scenario_dict, seed, ta, ta_kwargs,
                                 metalearning_configurations, n_jobs, dask_client,
                                 multi_objective_algorithm, multi_objective_kwargs):
            """随机搜索回调函数"""
            
            if n_jobs > 1 or (dask_client and len(dask_client.nthreads()) > 1):
                raise ValueError(
                    "请确保代码在 `if __name__ == '__main__'` 保护下运行"
                )
            
            # 配置纯随机搜索
            scenario_dict["minR"] = len(scenario_dict["instances"])
            scenario_dict["initial_incumbent"] = "RANDOM"
            scenario = Scenario(scenario_dict)
            
            return ROAR(
                scenario=scenario,
                rng=seed,
                tae_runner=ta,
                tae_runner_kwargs=ta_kwargs,
                run_id=seed,
                dask_client=dask_client,
                n_jobs=n_jobs,
            )
        
        return random_search_callback
    
    def create_roar_classifier(self, tmp_folder):
        """
        创建ROAR算法分类器
        
        参数:
            tmp_folder (str): 临时文件夹路径
            
        返回:
            AutoSklearnClassifier: 配置了ROAR的分类器
        """
        print(f"\n🏗️  创建ROAR分类器")
        
        classifier = autosklearn.classification.AutoSklearnClassifier(
            time_left_for_this_task=self.time_budget,
            per_run_time_limit=int(self.time_budget / 8),
            tmp_folder=tmp_folder + "_roar",
            
            # ROAR配置
            initial_configurations_via_metalearning=0,  # 禁用元学习
            get_smac_object_callback=self.get_roar_object_callback(),
            
            # 算法限制（加速搜索）
            include_estimators=['random_forest', 'extra_trees', 'gradient_boosting'],
            include_preprocessors=['no_preprocessing'],
            
            # 资源控制
            memory_limit=3072,
            n_jobs=1,
            delete_tmp_folder_after_terminate=False,
            seed=42
        )
        
        print(f"   ✅ ROAR分类器创建完成")
        return classifier
    
    def create_random_search_classifier(self, tmp_folder):
        """
        创建随机搜索分类器
        
        参数:
            tmp_folder (str): 临时文件夹路径
            
        返回:
            AutoSklearnClassifier: 配置了随机搜索的分类器
        """
        print(f"\n🏗️  创建随机搜索分类器")
        
        classifier = autosklearn.classification.AutoSklearnClassifier(
            time_left_for_this_task=self.time_budget,
            per_run_time_limit=int(self.time_budget / 8),
            tmp_folder=tmp_folder + "_random",
            
            # 随机搜索配置
            initial_configurations_via_metalearning=0,  # 禁用元学习
            get_smac_object_callback=self.get_random_search_callback(),
            
            # 算法限制（与ROAR保持一致）
            include_estimators=['random_forest', 'extra_trees', 'gradient_boosting'],
            include_preprocessors=['no_preprocessing'],
            
            # 资源控制
            memory_limit=3072,
            n_jobs=1,
            delete_tmp_folder_after_terminate=False,
            seed=42
        )
        
        print(f"   ✅ 随机搜索分类器创建完成")
        return classifier


class PerformanceTracker:
    """
    性能跟踪器
    
    跟踪和分析随机搜索过程中的性能表现，包括收敛分析、稳定性评估等。
    """
    
    def __init__(self):
        """初始化性能跟踪器"""
        self.roar_history = []
        self.random_history = []
        self.comparison_results = {}
        
    def track_training(self, algorithm_name, automl_model, X_test, y_test):
        """
        跟踪训练过程性能
        
        参数:
            algorithm_name (str): 算法名称
            automl_model: 训练好的AutoML模型
            X_test: 测试特征
            y_test: 测试标签
        """
        # 预测性能
        predictions = automl_model.predict(X_test)
        probabilities = automl_model.predict_proba(X_test)
        
        # 计算评估指标
        metrics = {
            'accuracy': sklearn.metrics.accuracy_score(y_test, predictions),
            'precision': sklearn.metrics.precision_score(y_test, predictions, average='weighted'),
            'recall': sklearn.metrics.recall_score(y_test, predictions, average='weighted'),
            'f1_score': sklearn.metrics.f1_score(y_test, predictions, average='weighted'),
            'auc': sklearn.metrics.roc_auc_score(y_test, probabilities[:, 1]),
            'confusion_matrix': confusion_matrix(y_test, predictions)
        }
        
        # 模型信息
        models = automl_model.show_models()
        model_info = {
            'model_count': len(models),
            'algorithm_diversity': len(set(type(model).__name__ for _, model in models.items())),
            'ensemble_weights': dict(models) if models else {}
        }
        
        # 统计信息
        stats = automl_model.sprint_statistics()
        
        # 记录结果
        result = {
            'algorithm': algorithm_name,
            'metrics': metrics,
            'model_info': model_info,
            'statistics': stats,
            'timestamp': datetime.now()
        }
        
        if algorithm_name == 'ROAR':
            self.roar_history.append(result)
        elif algorithm_name == 'Random':
            self.random_history.append(result)
        
        return result
    
    def compare_algorithms(self):
        """
        比较不同算法的性能
        
        返回:
            dict: 比较结果
        """
        if not self.roar_history or not self.random_history:
            print("⚠️  需要两种算法的结果才能进行比较")
            return {}
        
        roar_result = self.roar_history[-1]
        random_result = self.random_history[-1]
        
        comparison = {
            'accuracy_comparison': {
                'roar': roar_result['metrics']['accuracy'],
                'random': random_result['metrics']['accuracy'],
                'difference': roar_result['metrics']['accuracy'] - random_result['metrics']['accuracy']
            },
            'auc_comparison': {
                'roar': roar_result['metrics']['auc'],
                'random': random_result['metrics']['auc'],
                'difference': roar_result['metrics']['auc'] - random_result['metrics']['auc']
            },
            'model_complexity': {
                'roar_models': roar_result['model_info']['model_count'],
                'random_models': random_result['model_info']['model_count']
            },
            'algorithm_diversity': {
                'roar_diversity': roar_result['model_info']['algorithm_diversity'],
                'random_diversity': random_result['model_info']['algorithm_diversity']
            }
        }
        
        self.comparison_results = comparison
        return comparison


class SearchVisualizationAnalyzer:
    """
    搜索可视化分析器
    
    创建各种可视化图表分析随机搜索的性能和行为。
    """
    
    def __init__(self, output_dir="../results"):
        """
        初始化可视化分析器
        
        参数:
            output_dir (str): 输出目录
        """
        self.output_dir = output_dir
        os.makedirs(output_dir, exist_ok=True)
        
        # 设置图表风格
        plt.style.use('default')
        sns.set_palette("husl")
    
    def create_performance_comparison(self, roar_results, random_results):
        """
        创建性能对比图
        
        参数:
            roar_results (dict): ROAR结果
            random_results (dict): 随机搜索结果
        """
        fig, axes = plt.subplots(2, 2, figsize=(15, 10))
        fig.suptitle('ROAR vs 随机搜索性能对比', fontsize=16, fontweight='bold')
        
        # 1. 准确率对比
        algorithms = ['ROAR', '随机搜索']
        accuracies = [roar_results['metrics']['accuracy'], 
                     random_results['metrics']['accuracy']]
        
        axes[0, 0].bar(algorithms, accuracies, color=['#FF6B6B', '#4ECDC4'])
        axes[0, 0].set_title('准确率对比')
        axes[0, 0].set_ylabel('准确率')
        axes[0, 0].set_ylim(0.9, 1.0)
        for i, v in enumerate(accuracies):
            axes[0, 0].text(i, v + 0.002, f'{v:.4f}', ha='center', va='bottom')
        
        # 2. AUC对比
        aucs = [roar_results['metrics']['auc'], random_results['metrics']['auc']]
        axes[0, 1].bar(algorithms, aucs, color=['#FF6B6B', '#4ECDC4'])
        axes[0, 1].set_title('AUC对比')
        axes[0, 1].set_ylabel('AUC值')
        axes[0, 1].set_ylim(0.9, 1.0)
        for i, v in enumerate(aucs):
            axes[0, 1].text(i, v + 0.002, f'{v:.4f}', ha='center', va='bottom')
        
        # 3. 模型复杂度对比
        model_counts = [roar_results['model_info']['model_count'],
                       random_results['model_info']['model_count']]
        axes[1, 0].bar(algorithms, model_counts, color=['#FF6B6B', '#4ECDC4'])
        axes[1, 0].set_title('集成模型数量对比')
        axes[1, 0].set_ylabel('模型数量')
        for i, v in enumerate(model_counts):
            axes[1, 0].text(i, v + 0.1, str(v), ha='center', va='bottom')
        
        # 4. 算法多样性对比
        diversities = [roar_results['model_info']['algorithm_diversity'],
                      random_results['model_info']['algorithm_diversity']]
        axes[1, 1].bar(algorithms, diversities, color=['#FF6B6B', '#4ECDC4'])
        axes[1, 1].set_title('算法多样性对比')
        axes[1, 1].set_ylabel('算法类型数')
        for i, v in enumerate(diversities):
            axes[1, 1].text(i, v + 0.05, str(v), ha='center', va='bottom')
        
        plt.tight_layout()
        plt.savefig(os.path.join(self.output_dir, 'performance_comparison.png'), 
                   dpi=300, bbox_inches='tight')
        plt.show()
    
    def create_confusion_matrix_comparison(self, roar_results, random_results):
        """
        创建混淆矩阵对比图
        
        参数:
            roar_results (dict): ROAR结果
            random_results (dict): 随机搜索结果
        """
        fig, axes = plt.subplots(1, 2, figsize=(12, 5))
        fig.suptitle('混淆矩阵对比', fontsize=14, fontweight='bold')
        
        # ROAR混淆矩阵
        sns.heatmap(roar_results['metrics']['confusion_matrix'], 
                   annot=True, fmt='d', cmap='Blues',
                   xticklabels=['良性', '恶性'], 
                   yticklabels=['良性', '恶性'],
                   ax=axes[0])
        axes[0].set_title('ROAR算法')
        axes[0].set_xlabel('预测类别')
        axes[0].set_ylabel('真实类别')
        
        # 随机搜索混淆矩阵
        sns.heatmap(random_results['metrics']['confusion_matrix'], 
                   annot=True, fmt='d', cmap='Oranges',
                   xticklabels=['良性', '恶性'], 
                   yticklabels=['良性', '恶性'],
                   ax=axes[1])
        axes[1].set_title('随机搜索')
        axes[1].set_xlabel('预测类别')
        axes[1].set_ylabel('真实类别')
        
        plt.tight_layout()
        plt.savefig(os.path.join(self.output_dir, 'confusion_matrix_comparison.png'), 
                   dpi=300, bbox_inches='tight')
        plt.show()
    
    def create_metrics_radar_chart(self, roar_results, random_results):
        """
        创建指标雷达图
        
        参数:
            roar_results (dict): ROAR结果
            random_results (dict): 随机搜索结果
        """
        # 准备数据
        metrics = ['准确率', '精确率', '召回率', 'F1分数', 'AUC']
        roar_values = [
            roar_results['metrics']['accuracy'],
            roar_results['metrics']['precision'],
            roar_results['metrics']['recall'],
            roar_results['metrics']['f1_score'],
            roar_results['metrics']['auc']
        ]
        random_values = [
            random_results['metrics']['accuracy'],
            random_results['metrics']['precision'],
            random_results['metrics']['recall'],
            random_results['metrics']['f1_score'],
            random_results['metrics']['auc']
        ]
        
        # 创建雷达图
        angles = np.linspace(0, 2 * np.pi, len(metrics), endpoint=False).tolist()
        angles += angles[:1]  # 闭合图形
        
        roar_values += roar_values[:1]
        random_values += random_values[:1]
        
        fig, ax = plt.subplots(figsize=(10, 8), subplot_kw=dict(projection='polar'))
        
        # 绘制ROAR
        ax.plot(angles, roar_values, 'o-', linewidth=2, label='ROAR', color='#FF6B6B')
        ax.fill(angles, roar_values, alpha=0.25, color='#FF6B6B')
        
        # 绘制随机搜索
        ax.plot(angles, random_values, 'o-', linewidth=2, label='随机搜索', color='#4ECDC4')
        ax.fill(angles, random_values, alpha=0.25, color='#4ECDC4')
        
        # 设置标签
        ax.set_xticks(angles[:-1])
        ax.set_xticklabels(metrics)
        ax.set_ylim(0.85, 1.0)
        ax.set_title('性能指标雷达图', size=14, fontweight='bold', pad=20)
        ax.legend(loc='upper right', bbox_to_anchor=(1.3, 1.0))
        
        plt.tight_layout()
        plt.savefig(os.path.join(self.output_dir, 'metrics_radar_chart.png'), 
                   dpi=300, bbox_inches='tight')
        plt.show()


def main():
    """
    主函数：演示AutoML随机搜索策略
    """
    print("🎯 AutoML随机搜索策略示例")
    print("=" * 60)
    
    # 1. 数据加载和预处理
    print("\n📊 加载Wisconsin乳腺癌数据集")
    X, y = sklearn.datasets.load_breast_cancer(return_X_y=True)
    X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
        X, y, test_size=0.3, random_state=42, stratify=y
    )
    
    print(f"   数据集形状: {X.shape}")
    print(f"   训练集: {X_train.shape[0]}样本")
    print(f"   测试集: {X_test.shape[0]}样本")
    print(f"   特征数量: {X_train.shape[1]}个")
    print(f"   类别分布 - 训练集: 良性{np.sum(y_train==0)}, 恶性{np.sum(y_train==1)}")
    print(f"   类别分布 - 测试集: 良性{np.sum(y_test==0)}, 恶性{np.sum(y_test==1)}")
    
    # 2. 创建随机搜索控制器
    controller = RandomSearchController(
        time_budget=60,     # 1分钟时间预算
        search_budget=30    # 30次搜索评估
    )
    
    tmp_folder = f"/tmp/autosklearn_random_search_{int(time.time())}"
    
    # 3. 执行ROAR算法
    print(f"\n🔥 执行ROAR (Random Online Aggressive Racing) 算法")
    roar_classifier = controller.create_roar_classifier(tmp_folder)
    
    roar_start_time = time.time()
    roar_classifier.fit(X_train, y_train, dataset_name="breast_cancer")
    roar_end_time = time.time()
    
    print(f"   ✅ ROAR训练完成，耗时: {roar_end_time - roar_start_time:.2f}秒")
    
    # 4. 执行纯随机搜索
    print(f"\n🎲 执行纯随机搜索算法")
    random_classifier = controller.create_random_search_classifier(tmp_folder)
    
    random_start_time = time.time()
    random_classifier.fit(X_train, y_train, dataset_name="breast_cancer")
    random_end_time = time.time()
    
    print(f"   ✅ 随机搜索训练完成，耗时: {random_end_time - random_start_time:.2f}秒")
    
    # 5. 性能跟踪和分析
    tracker = PerformanceTracker()
    
    print(f"\n📈 性能评估和分析")
    
    # 跟踪ROAR性能
    roar_results = tracker.track_training('ROAR', roar_classifier, X_test, y_test)
    print(f"\n🔥 ROAR算法结果:")
    print(f"   准确率: {roar_results['metrics']['accuracy']:.4f}")
    print(f"   AUC: {roar_results['metrics']['auc']:.4f}")
    print(f"   集成模型数: {roar_results['model_info']['model_count']}")
    print(f"   算法多样性: {roar_results['model_info']['algorithm_diversity']}")
    
    # 跟踪随机搜索性能
    random_results = tracker.track_training('Random', random_classifier, X_test, y_test)
    print(f"\n🎲 随机搜索结果:")
    print(f"   准确率: {random_results['metrics']['accuracy']:.4f}")
    print(f"   AUC: {random_results['metrics']['auc']:.4f}")
    print(f"   集成模型数: {random_results['model_info']['model_count']}")
    print(f"   算法多样性: {random_results['model_info']['algorithm_diversity']}")
    
    # 算法对比分析
    comparison = tracker.compare_algorithms()
    print(f"\n⚖️  算法对比分析:")
    print(f"   准确率差异: {comparison['accuracy_comparison']['difference']:+.4f}")
    print(f"   AUC差异: {comparison['auc_comparison']['difference']:+.4f}")
    print(f"   模型复杂度对比: ROAR({comparison['model_complexity']['roar_models']}) vs Random({comparison['model_complexity']['random_models']})")
    
    # 6. 创建可视化分析
    print(f"\n📊 生成可视化分析图表")
    visualizer = SearchVisualizationAnalyzer("../results")
    
    # 性能对比图
    visualizer.create_performance_comparison(roar_results, random_results)
    
    # 混淆矩阵对比
    visualizer.create_confusion_matrix_comparison(roar_results, random_results)
    
    # 指标雷达图
    visualizer.create_metrics_radar_chart(roar_results, random_results)
    
    # 7. 统计显著性检验
    print(f"\n📐 统计显著性分析")
    accuracy_diff = roar_results['metrics']['accuracy'] - random_results['metrics']['accuracy']
    auc_diff = roar_results['metrics']['auc'] - random_results['metrics']['auc']
    
    print(f"   准确率提升: {accuracy_diff:+.4f}")
    print(f"   AUC提升: {auc_diff:+.4f}")
    
    if accuracy_diff > 0.01:
        print(f"   🎯 ROAR在准确率上有显著优势")
    elif accuracy_diff < -0.01:
        print(f"   🎲 随机搜索在准确率上有显著优势")
    else:
        print(f"   ⚖️  两种算法准确率相当")
    
    # 8. 生成详细报告
    print(f"\n📋 生成详细分析报告")
    
    # 显示模型详情
    print(f"\n🏆 ROAR最终集成模型:")
    pprint(roar_classifier.show_models(), indent=4)
    
    print(f"\n🎲 随机搜索最终集成模型:")
    pprint(random_classifier.show_models(), indent=4)
    
    # 训练统计信息
    print(f"\n📊 ROAR训练统计:")
    print(roar_classifier.sprint_statistics())
    
    print(f"\n📊 随机搜索训练统计:")
    print(random_classifier.sprint_statistics())
    
    # 9. 生成最终总结
    total_time = (roar_end_time - roar_start_time) + (random_end_time - random_start_time)
    
    print(f"\n🎉 随机搜索策略对比分析完成!")
    print(f"   总执行时间: {total_time:.2f}秒")
    print(f"   ROAR最终准确率: {roar_results['metrics']['accuracy']:.4f}")
    print(f"   随机搜索最终准确率: {random_results['metrics']['accuracy']:.4f}")
    print(f"   性能提升: {accuracy_diff:+.4f}")
    
    # 推荐使用场景
    print(f"\n💡 使用建议:")
    if accuracy_diff > 0.005:
        print(f"   推荐使用ROAR算法，在该数据集上表现更优")
    elif accuracy_diff < -0.005:
        print(f"   推荐使用纯随机搜索，计算更简单且效果不差")
    else:
        print(f"   两种算法性能相当，可根据计算资源和复杂度需求选择")
    
    # 10. 清理建议
    print(f"\n🧹 资源清理:")
    print(f"   ROAR临时文件: {tmp_folder}_roar")
    print(f"   随机搜索临时文件: {tmp_folder}_random")
    print(f"   可视化结果: ../results/")
    
    return controller, tracker, roar_results, random_results, comparison


if __name__ == "__main__":
    # 设置随机种子以确保结果可重现
    np.random.seed(42)
    
    try:
        controller, tracker, roar_results, random_results, comparison = main()
        print(f"\n✅ 程序执行成功!")
        
    except KeyboardInterrupt:
        print(f"\n⚠️  程序被用户中断")
        
    except Exception as e:
        print(f"\n❌ 程序执行失败: {str(e)}")
        import traceback
        traceback.print_exc()
        
    finally:
        print(f"\n👋 感谢使用AutoML随机搜索示例!")